{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from deeptables.models import deeptable,deepnets\n",
    "import pandas as pd\n",
    "from sklearn.metrics import roc_auc_score\n",
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start cross validation\n",
      "2 class detected, {0, 1}, so inferred as a [binary classification] task\n",
      "Preparing features taken 0.0024428367614746094s\n",
      "Imputation taken 0.03056812286376953s\n",
      "Categorical encoding taken 0.014766931533813477s\n",
      "Discretization taken 0.0456697940826416s\n",
      "transform X_test\n",
      "Iterators:KFold(n_splits=5, random_state=9527, shuffle=True)\n",
      "Injected a callback [EarlyStopping]. monitor:val_AUC, patience:1, mode:max\n",
      "\n",
      "Fold:1\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/jack/opt/anaconda3/envs/tf_2_0/lib/python3.7/site-packages/sklearn/preprocessing/_discretization.py:202: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 0 are removed. Consider decreasing the number of bins.\n",
      "  'decreasing the number of bins.' % jj)\n",
      "/Users/jack/opt/anaconda3/envs/tf_2_0/lib/python3.7/site-packages/sklearn/preprocessing/_discretization.py:202: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 0 are removed. Consider decreasing the number of bins.\n",
      "  'decreasing the number of bins.' % jj)\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ">>>>>>>>>>>>>>>>>>>>>> Model Desc <<<<<<<<<<<<<<<<<<<<<<< \n",
      "---------------------------------------------------------\n",
      "inputs:\n",
      "---------------------------------------------------------\n",
      "['all_categorical_vars: (9)', 'input_continuous_all: (4)']\n",
      "---------------------------------------------------------\n",
      "embeddings:\n",
      "---------------------------------------------------------\n",
      "input_dims: [4, 5, 10, 6, 4, 5, 4, 4, 6]\n",
      "output_dims: [4, 4, 4, 4, 4, 4, 4, 4, 4]\n",
      "dropout: 0.3\n",
      "---------------------------------------------------------\n",
      "dense: dropout: 0\n",
      "batch_normalization: False\n",
      "---------------------------------------------------------\n",
      "concat_embed_dense: shape: (None, 40)\n",
      "---------------------------------------------------------\n",
      "nets: ['dcn_nets']\n",
      "---------------------------------------------------------\n",
      "dcn-widecross: input_shape (None, 40), output_shape (None, 40)\n",
      "dcn-dnn2: input_shape (None, 40), output_shape (None, 64)\n",
      "dcn: input_shape (None, 40), output_shape (None, 104)\n",
      "---------------------------------------------------------\n",
      "stacking_op: add\n",
      "---------------------------------------------------------\n",
      "output: activation: sigmoid, output_shape: (None, 1), use_bias: True\n",
      "loss: binary_crossentropy\n",
      "optimizer: Adam\n",
      "---------------------------------------------------------\n",
      "\n",
      "Train on 501 samples, validate on 126 samples\n",
      "Epoch 1/50\n",
      "501/501 [==============================] - 18s 37ms/sample - loss: 0.9521 - AUC: 0.5750 - val_loss: 37.7395 - val_AUC: 0.4820\n",
      "Epoch 2/50\n",
      "501/501 [==============================] - 1s 2ms/sample - loss: 0.7137 - AUC: 0.6968 - val_loss: 7.2463 - val_AUC: 0.5875\n",
      "Epoch 3/50\n",
      "501/501 [==============================] - 1s 2ms/sample - loss: 0.5636 - AUC: 0.7661 - val_loss: 2.8878 - val_AUC: 0.6155\n",
      "Epoch 4/50\n",
      "501/501 [==============================] - 1s 2ms/sample - loss: 0.5142 - AUC: 0.8140 - val_loss: 1.7284 - val_AUC: 0.6462\n",
      "Epoch 5/50\n",
      "501/501 [==============================] - 1s 2ms/sample - loss: 0.4950 - AUC: 0.8258 - val_loss: 1.1784 - val_AUC: 0.6852\n",
      "Epoch 6/50\n",
      "501/501 [==============================] - 1s 2ms/sample - loss: 0.5036 - AUC: 0.8180 - val_loss: 0.9231 - val_AUC: 0.7156\n",
      "Epoch 7/50\n",
      "501/501 [==============================] - 1s 2ms/sample - loss: 0.4823 - AUC: 0.8432 - val_loss: 0.8205 - val_AUC: 0.7168\n",
      "Epoch 8/50\n",
      "480/501 [===========================>..] - ETA: 0s - loss: 0.4418 - AUC: 0.8609Restoring model weights from the end of the best epoch.\n",
      "501/501 [==============================] - 1s 2ms/sample - loss: 0.4451 - AUC: 0.8575 - val_loss: 0.7930 - val_AUC: 0.7113\n",
      "Epoch 00008: early stopping\n",
      "Fold 1 fitting over.\n",
      "Fold 1 scoring over.\n",
      "Save model to:dt_output/dt_20201019 132634_dcn_nets/dcn_nets-kfold-1.h5.\n",
      "\n",
      "Fold:2\n",
      "\n",
      ">>>>>>>>>>>>>>>>>>>>>> Model Desc <<<<<<<<<<<<<<<<<<<<<<< \n",
      "---------------------------------------------------------\n",
      "inputs:\n",
      "---------------------------------------------------------\n",
      "['all_categorical_vars: (9)', 'input_continuous_all: (4)']\n",
      "---------------------------------------------------------\n",
      "embeddings:\n",
      "---------------------------------------------------------\n",
      "input_dims: [4, 5, 10, 6, 4, 5, 4, 4, 6]\n",
      "output_dims: [4, 4, 4, 4, 4, 4, 4, 4, 4]\n",
      "dropout: 0.3\n",
      "---------------------------------------------------------\n",
      "dense: dropout: 0\n",
      "batch_normalization: False\n",
      "---------------------------------------------------------\n",
      "concat_embed_dense: shape: (None, 40)\n",
      "---------------------------------------------------------\n",
      "nets: ['dcn_nets']\n",
      "---------------------------------------------------------\n",
      "dcn-widecross: input_shape (None, 40), output_shape (None, 40)\n",
      "dcn-dnn2: input_shape (None, 40), output_shape (None, 64)\n",
      "dcn: input_shape (None, 40), output_shape (None, 104)\n",
      "---------------------------------------------------------\n",
      "stacking_op: add\n",
      "---------------------------------------------------------\n",
      "output: activation: sigmoid, output_shape: (None, 1), use_bias: True\n",
      "loss: binary_crossentropy\n",
      "optimizer: Adam\n",
      "---------------------------------------------------------\n",
      "\n",
      "Train on 501 samples, validate on 126 samples\n",
      "Epoch 1/50\n",
      "501/501 [==============================] - 26s 52ms/sample - loss: 0.7795 - AUC: 0.5128 - val_loss: 1.1275 - val_AUC: 0.6558\n",
      "Epoch 2/50\n",
      "501/501 [==============================] - 0s 805us/sample - loss: 0.5913 - AUC: 0.7301 - val_loss: 0.5893 - val_AUC: 0.7645\n",
      "Epoch 3/50\n",
      "501/501 [==============================] - 0s 837us/sample - loss: 0.5118 - AUC: 0.8164 - val_loss: 0.5747 - val_AUC: 0.7646\n",
      "Epoch 4/50\n",
      "416/501 [=======================>......] - ETA: 0s - loss: 0.4809 - AUC: 0.8314Restoring model weights from the end of the best epoch.\n",
      "501/501 [==============================] - 0s 823us/sample - loss: 0.4838 - AUC: 0.8285 - val_loss: 0.5840 - val_AUC: 0.7552\n",
      "Epoch 00004: early stopping\n",
      "Fold 2 fitting over.\n",
      "Fold 2 scoring over.\n",
      "Save model to:dt_output/dt_20201019 132634_dcn_nets/dcn_nets-kfold-2.h5.\n",
      "\n",
      "Fold:3\n",
      "\n",
      ">>>>>>>>>>>>>>>>>>>>>> Model Desc <<<<<<<<<<<<<<<<<<<<<<< \n",
      "---------------------------------------------------------\n",
      "inputs:\n",
      "---------------------------------------------------------\n",
      "['all_categorical_vars: (9)', 'input_continuous_all: (4)']\n",
      "---------------------------------------------------------\n",
      "embeddings:\n",
      "---------------------------------------------------------\n",
      "input_dims: [4, 5, 10, 6, 4, 5, 4, 4, 6]\n",
      "output_dims: [4, 4, 4, 4, 4, 4, 4, 4, 4]\n",
      "dropout: 0.3\n",
      "---------------------------------------------------------\n",
      "dense: dropout: 0\n",
      "batch_normalization: False\n",
      "---------------------------------------------------------\n",
      "concat_embed_dense: shape: (None, 40)\n",
      "---------------------------------------------------------\n",
      "nets: ['dcn_nets']\n",
      "---------------------------------------------------------\n",
      "dcn-widecross: input_shape (None, 40), output_shape (None, 40)\n",
      "dcn-dnn2: input_shape (None, 40), output_shape (None, 64)\n",
      "dcn: input_shape (None, 40), output_shape (None, 104)\n",
      "---------------------------------------------------------\n",
      "stacking_op: add\n",
      "---------------------------------------------------------\n",
      "output: activation: sigmoid, output_shape: (None, 1), use_bias: True\n",
      "loss: binary_crossentropy\n",
      "optimizer: Adam\n",
      "---------------------------------------------------------\n",
      "\n",
      "Train on 502 samples, validate on 125 samples\n",
      "Epoch 1/50\n",
      "502/502 [==============================] - 13s 26ms/sample - loss: 0.8884 - AUC: 0.4606 - val_loss: 6.2443 - val_AUC: 0.5613\n",
      "Epoch 2/50\n",
      "502/502 [==============================] - 0s 959us/sample - loss: 0.6566 - AUC: 0.6906 - val_loss: 1.2308 - val_AUC: 0.6669\n",
      "Epoch 3/50\n",
      "416/502 [=======================>......] - ETA: 0s - loss: 0.5611 - AUC: 0.7771Restoring model weights from the end of the best epoch.\n",
      "502/502 [==============================] - 0s 936us/sample - loss: 0.5502 - AUC: 0.7823 - val_loss: 0.7232 - val_AUC: 0.6203\n",
      "Epoch 00003: early stopping\n",
      "Fold 3 fitting over.\n",
      "Fold 3 scoring over.\n",
      "Save model to:dt_output/dt_20201019 132634_dcn_nets/dcn_nets-kfold-3.h5.\n",
      "\n",
      "Fold:4\n",
      "\n",
      ">>>>>>>>>>>>>>>>>>>>>> Model Desc <<<<<<<<<<<<<<<<<<<<<<< \n",
      "---------------------------------------------------------\n",
      "inputs:\n",
      "---------------------------------------------------------\n",
      "['all_categorical_vars: (9)', 'input_continuous_all: (4)']\n",
      "---------------------------------------------------------\n",
      "embeddings:\n",
      "---------------------------------------------------------\n",
      "input_dims: [4, 5, 10, 6, 4, 5, 4, 4, 6]\n",
      "output_dims: [4, 4, 4, 4, 4, 4, 4, 4, 4]\n",
      "dropout: 0.3\n",
      "---------------------------------------------------------\n",
      "dense: dropout: 0\n",
      "batch_normalization: False\n",
      "---------------------------------------------------------\n",
      "concat_embed_dense: shape: (None, 40)\n",
      "---------------------------------------------------------\n",
      "nets: ['dcn_nets']\n",
      "---------------------------------------------------------\n",
      "dcn-widecross: input_shape (None, 40), output_shape (None, 40)\n",
      "dcn-dnn2: input_shape (None, 40), output_shape (None, 64)\n",
      "dcn: input_shape (None, 40), output_shape (None, 104)\n",
      "---------------------------------------------------------\n",
      "stacking_op: add\n",
      "---------------------------------------------------------\n",
      "output: activation: sigmoid, output_shape: (None, 1), use_bias: True\n",
      "loss: binary_crossentropy\n",
      "optimizer: Adam\n",
      "---------------------------------------------------------\n",
      "\n",
      "Train on 502 samples, validate on 125 samples\n",
      "Epoch 1/50\n",
      "502/502 [==============================] - 18s 36ms/sample - loss: 0.7082 - AUC: 0.6263 - val_loss: 2.0343 - val_AUC: 0.6317\n",
      "Epoch 2/50\n",
      "502/502 [==============================] - 0s 718us/sample - loss: 0.5328 - AUC: 0.7971 - val_loss: 0.7405 - val_AUC: 0.7166\n",
      "Epoch 3/50\n",
      "502/502 [==============================] - 1s 1ms/sample - loss: 0.4934 - AUC: 0.8227 - val_loss: 0.6384 - val_AUC: 0.7572\n",
      "Epoch 4/50\n",
      "502/502 [==============================] - 1s 1ms/sample - loss: 0.4755 - AUC: 0.8371 - val_loss: 0.6225 - val_AUC: 0.7852\n",
      "Epoch 5/50\n",
      "384/502 [=====================>........] - ETA: 0s - loss: 0.4430 - AUC: 0.8556Restoring model weights from the end of the best epoch.\n",
      "502/502 [==============================] - 0s 976us/sample - loss: 0.4520 - AUC: 0.8511 - val_loss: 0.6182 - val_AUC: 0.7719\n",
      "Epoch 00005: early stopping\n",
      "Fold 4 fitting over.\n",
      "Fold 4 scoring over.\n",
      "Save model to:dt_output/dt_20201019 132634_dcn_nets/dcn_nets-kfold-4.h5.\n",
      "\n",
      "Fold:5\n",
      "\n",
      ">>>>>>>>>>>>>>>>>>>>>> Model Desc <<<<<<<<<<<<<<<<<<<<<<< \n",
      "---------------------------------------------------------\n",
      "inputs:\n",
      "---------------------------------------------------------\n",
      "['all_categorical_vars: (9)', 'input_continuous_all: (4)']\n",
      "---------------------------------------------------------\n",
      "embeddings:\n",
      "---------------------------------------------------------\n",
      "input_dims: [4, 5, 10, 6, 4, 5, 4, 4, 6]\n",
      "output_dims: [4, 4, 4, 4, 4, 4, 4, 4, 4]\n",
      "dropout: 0.3\n",
      "---------------------------------------------------------\n",
      "dense: dropout: 0\n",
      "batch_normalization: False\n",
      "---------------------------------------------------------\n",
      "concat_embed_dense: shape: (None, 40)\n",
      "---------------------------------------------------------\n",
      "nets: ['dcn_nets']\n",
      "---------------------------------------------------------\n",
      "dcn-widecross: input_shape (None, 40), output_shape (None, 40)\n",
      "dcn-dnn2: input_shape (None, 40), output_shape (None, 64)\n",
      "dcn: input_shape (None, 40), output_shape (None, 104)\n",
      "---------------------------------------------------------\n",
      "stacking_op: add\n",
      "---------------------------------------------------------\n",
      "output: activation: sigmoid, output_shape: (None, 1), use_bias: True\n",
      "loss: binary_crossentropy\n",
      "optimizer: Adam\n",
      "---------------------------------------------------------\n",
      "\n",
      "Train on 502 samples, validate on 125 samples\n",
      "Epoch 1/50\n",
      "502/502 [==============================] - 12s 24ms/sample - loss: 0.8035 - AUC: 0.5961 - val_loss: 0.7553 - val_AUC: 0.6130\n",
      "Epoch 2/50\n",
      "502/502 [==============================] - 1s 1ms/sample - loss: 0.5496 - AUC: 0.7869 - val_loss: 0.6531 - val_AUC: 0.7746\n",
      "Epoch 3/50\n",
      "502/502 [==============================] - 0s 763us/sample - loss: 0.4983 - AUC: 0.8199 - val_loss: 0.6347 - val_AUC: 0.8105\n",
      "Epoch 4/50\n",
      "416/502 [=======================>......] - ETA: 0s - loss: 0.4626 - AUC: 0.8518Restoring model weights from the end of the best epoch.\n",
      "502/502 [==============================] - 0s 976us/sample - loss: 0.4588 - AUC: 0.8494 - val_loss: 0.6509 - val_AUC: 0.7811\n",
      "Epoch 00004: early stopping\n",
      "Fold 5 fitting over.\n",
      "Fold 5 scoring over.\n",
      "Save model to:dt_output/dt_20201019 132634_dcn_nets/dcn_nets-kfold-5.h5.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:  2.0min finished\n"
     ]
    }
   ],
   "source": [
    "dftrain = pd.read_csv('https://storage.googleapis.com/tf-datasets/titanic/train.csv')\n",
    "dfeval = pd.read_csv('https://storage.googleapis.com/tf-datasets/titanic/eval.csv')\n",
    "y_train = dftrain.pop('survived')\n",
    "y_eval = dfeval.pop('survived')\n",
    "\n",
    "config = deeptable.ModelConfig(nets=deepnets.DCN,auto_discrete=True,metrics=['AUC'])\n",
    "\n",
    "dt = deeptable.DeepTable(config=config)\n",
    "oof_proba, eval_proba_mean, test_proba_mean = dt.fit_cross_validation(\n",
    "    dftrain, \n",
    "    y_train, \n",
    "    X_test=dfeval, \n",
    "    num_folds=5, \n",
    "    epochs=50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>type</th>\n",
       "      <th>loss</th>\n",
       "      <th>*auc</th>\n",
       "      <th>val_loss</th>\n",
       "      <th>val_auc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>dcn_nets-kfold-1</td>\n",
       "      <td>val</td>\n",
       "      <td>0.445111</td>\n",
       "      <td>0.857538</td>\n",
       "      <td>0.793008</td>\n",
       "      <td>0.711317</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>dcn_nets-kfold-4</td>\n",
       "      <td>val</td>\n",
       "      <td>0.452028</td>\n",
       "      <td>0.851079</td>\n",
       "      <td>0.618186</td>\n",
       "      <td>0.771930</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>dcn_nets-kfold-5</td>\n",
       "      <td>val</td>\n",
       "      <td>0.458760</td>\n",
       "      <td>0.849411</td>\n",
       "      <td>0.650901</td>\n",
       "      <td>0.781089</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>dcn_nets-kfold-2</td>\n",
       "      <td>val</td>\n",
       "      <td>0.483778</td>\n",
       "      <td>0.828541</td>\n",
       "      <td>0.583975</td>\n",
       "      <td>0.755244</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>dcn_nets-kfold-3</td>\n",
       "      <td>val</td>\n",
       "      <td>0.550232</td>\n",
       "      <td>0.782319</td>\n",
       "      <td>0.723225</td>\n",
       "      <td>0.620265</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              model type      loss      *auc  val_loss   val_auc\n",
       "0  dcn_nets-kfold-1  val  0.445111  0.857538  0.793008  0.711317\n",
       "1  dcn_nets-kfold-4  val  0.452028  0.851079  0.618186  0.771930\n",
       "2  dcn_nets-kfold-5  val  0.458760  0.849411  0.650901  0.781089\n",
       "3  dcn_nets-kfold-2  val  0.483778  0.828541  0.583975  0.755244\n",
       "4  dcn_nets-kfold-3  val  0.550232  0.782319  0.723225  0.620265"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dt.leaderboard"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7150098593964335"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# oof score\n",
    "roc_auc_score(y_train, oof_proba)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7259565350474442"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# test score\n",
    "roc_auc_score(y_eval,test_proba_mean)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
